# Define your item pipelines here
#
import json

import pymongo
import redis
import scrapy
import csv
from itemadapter import ItemAdapter
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface

# class TutorialPipeline:
#     def open_spider(self, spider):
#         self.client = pymongo.MongoClient(host='localhost', port=27017)
#         self.db = self.client['quotes']
#         self.collection = self.db['quotes_all']
#
#     def process_item(self, item, spider):
#         if spider.name == 'quotes_all':
#             print(spider.name,"-------------------------------")
#             self.collection.insert_one(item)
#         return item
#
#
#     def close_spider(self, spider):
#         self.client.close()





# class TutorialPipeline:
#
#
#     def process_item(self, item, spider):
#         return item


class ProducerPipeline:
    def open_spider(self, spider):
        self.client =redis.Redis()
        self.client.delete("tasks")

    def process_item(self, item, spider):
        self.client.lpush("tasks", json.dumps(item, ensure_ascii=False))
        return item

    def close_spider(self, spider):
        self.client.close()



# 美食csv
# class CSVPipeline:
#     def open_spider(self, spider):
#         if spider.name == "food":
#             self.start_id = 101
#             self.f = open(f"{spider.name}.csv", "w", newline="")
#             self.csv_writer = csv.writer(self.f)
#             self.csv_writer.writerow(["id", "菜名", "图片地址", "作家"])
#
#             self.items = []
#
#     def process_item(self, item, spider):
#         if spider.name == "food":
#             self.start_id += 1
#             self.items.append([self.start_id, item['菜名'], item['图片地址'], item['作家']])
#             if len(self.items) == 15:
#                 self.csv_writer.writerows(self.items)
#                 self.items.clear()
#         return item
#
#     def close_spider(self, spider):
#         if spider.name == "food":
#             if len(self.items) > 0:
#                 self.csv_writer.writerows(self.items)
#                 self.items.clear()
#             self.f.close()








# class MySqlPipeline:
#     def open_spider(self, spider):
#         pass
#
#     def process_item(self, item, spider):
#         # print(item, type(item))
#         print(item['author'], item['text'], item['tags'])
#         return item
#
#     def close_spider(self, spider):
#         pass




# class CSVPipeline:
#     def open_spider(self, spider):
#         print(f"open  CSVPipeline ")
#         if spider.name == "quotes_all":
#             self.start_id = 100
#             self.f = open(f"{spider.name}.csv", "w", newline="")
#             self.csv_writer = csv.writer(self.f)
#             self.csv_writer.writerow(["id", "author", "text", "tags"])
#
#             self.items = []
#
#     def process_item(self, item, spider):
#         if spider.name == "quotes_all":
#             self.start_id += 1
#             self.items.append([self.start_id,   item['author'], item['text'], "|".join(item['tags']),  ])
#             if len(self.items) == 15:
#                 self.csv_writer.writerows(self.items)
#                 self.items.clear()
#         return item
#
#     def close_spider(self, spider):
#         if spider.name == "quotes_all":
#             if len(self.items) > 0:
#                 self.csv_writer.writerows(self.items)
#                 self.items.clear()
#             self.f.close()
#
# class MongolPipeline:
#     def open_spider(self, spider):
#         print(f"open  MongolPipeline ")
#         if spider.name == "quotes_all":
#             self.client = pymongo.MongoClient("mongodb://localhost:27017/")
#             self.db = self.client.get_database("quotes")
#             self.collection = self.db.get_collection("quotes")
#
#             self.items = []
#
#     def process_item(self, item, spider):
#         # print(dir(spider))
#         if spider.name == "quotes_all":
#             # print(spider.name, "--------------------------", item)
#             # self.collection.insert_one(item)
#             self.items.append(item)
#             if len(self.items) == 15:
#                 self.collection.insert_many(self.items)
#                 self.items.clear()
#         return item
#
#     def close_spider(self, spider):
#         print(f"close  MongolPipeline ")
#         if spider.name == "quotes_all":
#             if len(self.items) > 0:
#                 self.collection.insert_many(self.items)
#                 self.items.clear()
#             self.client.close()

